package com.eric.r2d2.pageProcessor;

import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.net.InetAddress;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.UnknownHostException;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashSet;
import java.util.List;
import java.util.Properties;
import java.util.concurrent.atomic.AtomicLong;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.eric.r2d2.utils.GetRoot;

import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.selector.Selectable;

public class SeoInfoExtractor implements PageProcessor {

	private AtomicLong pageCount = new AtomicLong(0);;

	// Seconds
	private long total_time_spent = 0;

	// 用于诊断页面下载和处理速度
	long a;

	int PAGE_TEXT_COUNT_LOWER_LIMIT;
	int URL_LENGTH_LIMIT;
	int LINKS_LIMIT;
	String EXCLUDE_FILE_REGEX;

	public SeoInfoExtractor() {
		a = System.currentTimeMillis();
		try {
			p.load(new FileInputStream(new File("resources/SEO_INFO_SETTINGS")));
		} catch (FileNotFoundException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
			logger.error("SEO INFO SETTING File Not Found");
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
		PAGE_TEXT_COUNT_LOWER_LIMIT = Integer.parseInt(p.getProperty("PAGE_TEXT_COUNT_LOWER_LIMIT"));
		URL_LENGTH_LIMIT = Integer.parseInt(p.getProperty("URL_LENGTH_LIMIT"));
		EXCLUDE_FILE_REGEX = p.getProperty("EXCLUDE_FILE_REGEX");
		LINKS_LIMIT = Integer.parseInt(p.getProperty("LINKS_LIMIT"));
	}

	public Site getSite() {
		// TODO Auto-generated method stub
		return site;
	}

	private final String UA = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_2) AppleWebKit/537.31 (KHTML, like Gecko) Chrome/26.0.1410.65 Safari/537.31";

	private Site site = Site.me().setUserAgent(UA);

	private boolean crawlExternal = false;

	public void crawlExternal() {
		crawlExternal = true;
	}

	private boolean recrawl = false;

	public void recrawl() {
		recrawl = true;
	}

	private Logger logger = LoggerFactory.getLogger(getClass());

	Properties p = new Properties();

	public void process(Page page) {

		// URL的域名判断，方便定义是否爬行站外
		String root = null;
		URL url = null;
		try {
			url = new URL(page.getUrl().toString());
			root = GetRoot.getRoot(page.getUrl().toString());
		} catch (MalformedURLException e) {
			e.printStackTrace();
			logger.info("[INFO]Malformed URL:" + e.getMessage() + " " + page.getUrl());
		}

		// 添加URL至爬行队列
		List<String> requests = new ArrayList<String>();
		if (recrawl) {
			// 不添加任何新url到队列，只从scheduler读取
		} else {
			if (crawlExternal) {
				List<String> r = page.getHtml().links().regex(".+").all();
				for (String s : r) {
					// URL格式和长度判断，不爬取图片，不爬取超过255字符的
					if (!s.contains("#") && !s.matches(EXCLUDE_FILE_REGEX) &&
					// !s.matches(".*\\.(bmp|jpg|tiff|gif|pcx|tga|exif|fpx|svg|psd|cdr|pcd|dxf|ufo|eps|ai|raw"
					// +
					// "|BMP|JPG|TIFF|GIF|PCX|TGA|EXIF|FPX|SVG|PSD|CDR|PCD|DXF|UFO|EPS|AI|RAW|"
					// +
					// "Bmp|Jpg|Tiff|Gif|Pcx|Tga|Exif|Fpx|Svg|Psd|Cdr|Pcd|Dxf|Ufo|Eps|Ai|Raw)$")
					// &&
					// !s.matches(".*\\.(rar|zip|tar|pdf)$") &&
							!s.matches(".*?(http.*){2,}.*?") && !s.contains("mailto") && s.length() < URL_LENGTH_LIMIT)
						requests.add(s);

				}
			} else {
				List<String> r = page.getHtml().links().regex("https?://\\w+\\." + root + "(/[^?&#]*)?").all();
				for (String s : r) {
					if (!s.contains("#") && !s.matches(EXCLUDE_FILE_REGEX) && !s.contains("mailto")
							&& s.length() < URL_LENGTH_LIMIT)
						requests.add(s);
				}
			}
			page.addTargetRequests(requests);
		}

		Selectable html = page.getHtml();
		JSONObject results = new JSONObject();

		// 提取网页的一些基本信息
		results.put("url", page.getUrl().toString());
		results.put("url_hash", page.getUrl().toString().hashCode());
		results.put("protocol", url.getProtocol());
		results.put("root_domain", root);
		results.put("domain", url.getHost());

		// 爬取的时间
		// 因为是写到JSON，后续再导入mongodb，所以保存日期的方法与直接写入mongodb不同（put("date",new
		// Date())即可）
		// results.put("crawl_date", new SimpleDateFormat("yyyyMMdd").format(new
		// Date()));
		JSONObject json = new JSONObject();
		json.put("$date", new Date().getTime());
		results.put("crawl_date", json);

		// 提取Title
		JSONObject titleJ = new JSONObject().put("content", html.xpath("//title").replace("<[^>]*?>", "").toString());
		titleJ.put("length", html.xpath("//title").toString().length());
		results.put("title", titleJ);

		// 提取Description
		JSONObject descriptionJ = new JSONObject().put("content", html.xpath("//meta[@name='description']/@content"));
		try {
			if (html.xpath("//meta[@name='description']/@content") != null)
				descriptionJ.put("length", html.xpath("//meta[@name='description']/@content").toString().length());
		} catch (Exception e) {

		}
		results.put("description", descriptionJ);

		// 提取H1
		JSONObject h1J = new JSONObject().put("content", html.xpath("//h1").replace("<[^>]*?>", "").toString());
		if (html.xpath("//h1").toString() != null)
			h1J.put("length", html.xpath("//h1").replace("<[^>]*?>", "").toString().length());
		results.put("h1", h1J);

		// 提取H2
		JSONObject h2J = new JSONObject().put("content", html.xpath("//h2").replace("<[^>]*?>", "").all());
		results.put("h2", h2J);

		// 提取页面链接，超过LINKS_LIMIT的不保存
		JSONArray outlinksJA = new JSONArray();
		List<String> aBlockList = new ArrayList<String>(new HashSet<String>(html.xpath("//a").all()));
		int links_count = 0;
		for (String aBlock : aBlockList) {
			if (links_count > LINKS_LIMIT)
				break;
			JSONObject outlinksJ = new JSONObject();
			Pattern p = Pattern.compile("(?<=href=\")https?://.*?\\..*?\\..*?(/.*?)?(?=\")");
			Matcher m = p.matcher(aBlock);

			String out_link;
			if (m.find()) {
				out_link = m.group().replaceAll("amp;", "");

				if (!out_link.contains("#") && !out_link.matches(EXCLUDE_FILE_REGEX) && !out_link.contains("mailto")
						&& out_link.length() < URL_LENGTH_LIMIT) {
					outlinksJ.put("url", out_link);

					if (aBlock.contains("src=")) {
						outlinksJ.put("type", "img");
					} else {
						outlinksJ.put("type", "txt");
					}
					if (aBlock.contains("nofollow")) {
						outlinksJ.put("nofollow", "yes");
					} else {
						outlinksJ.put("nofollow", "no");
					}
					if (m.group().contains(root)) {
						outlinksJ.put("ext", "no");
					} else {
						outlinksJ.put("ext", "yes");
					}
					Elements aBlock_doc = Jsoup.parse(aBlock).getElementsByTag("a");
					outlinksJ.put("anchor", aBlock_doc.text());
					outlinksJ.put("title", aBlock_doc.attr("title"));

					outlinksJA.put(outlinksJ);
				}
				links_count++;
			}

		}
		results.put("links", outlinksJA);

		// 计算链接数量
		results.put("outlinks_count", aBlockList.size());

		// 计算出站链接数量
		JSONArray ja = results.getJSONArray("links");
		int x = 0;
		for (int i = 0; i < ja.length(); i++) {
			JSONObject jo = ja.getJSONObject(i);
			if (jo.get("ext") == "yes")
				x++;
		}
		results.put("ext_outlinks_count", x);

		// 提取返回码
		results.put("status", page.getStatusCode());

		// 提取正文字数，并计算simhash
		Document doc = Jsoup.parse(html.toString());
		Elements elements = doc.getAllElements();
		StringBuffer sb = new StringBuffer();
		for (Element e : elements) {
			sb.append(e.ownText().trim());
		}
		results.put("text_count", sb.length());
		results.put("text_simhash", new SimHash(sb.toString(), 64).simHash());

		// 保存爬虫hostname
		try {
			results.put("crawler", InetAddress.getLocalHost());
		} catch (JSONException e1) {
			// TODO Auto-generated catch block
			e1.printStackTrace();
		} catch (UnknownHostException e1) {
			// TODO Auto-generated catch block
			e1.printStackTrace();
		}

		// 存放最终结果，字数少于PAGE_TEXT_COUNT_LOWER_LIMIT的不保存
		if (sb.length() > PAGE_TEXT_COUNT_LOWER_LIMIT) {
			page.putField("results", results);
			long b = ((System.currentTimeMillis() - a)) / 1000;
			logger.info("Page Extracted In " + b + " seconds: " + page.getUrl());
			total_time_spent = total_time_spent + b;
			logger.info("Total Page Extracted:" + pageCount.incrementAndGet() + " Total Seconds:" + total_time_spent);
			a = System.currentTimeMillis();
		} else {
			page.getResultItems().setSkip(true);
			logger.info("Text Less Than " + PAGE_TEXT_COUNT_LOWER_LIMIT + " Characters,Pass: " + page.getUrl());
		}
	}
}
